package com.stegniy.webCrawler;

import java.io.IOException;
import java.net.URL;
import java.util.LinkedHashSet;

public class Main {
    public static void main(String[] args) throws IOException {
        String arg = "http://dou.ua";
        //String arg = args[0];
        int depth = 2;
        int i = 0;
        URL url = new URL(arg);

        WebCrawler crawler = new WebCrawler();

        String s = crawler.getPage(url);
        crawler.downloadPage(s);

        LinkedHashSet<String> set = crawler.getLinks(url, s);

        for (String str : set) {
            URL url1 = new URL(str);
            crawler.downloadPage(crawler.getPage(url1));
        }

        System.out.println(set);
    }
}
